serve --long-model: short conversations on -m's route, long ones on W4A4 - #203

    const size_t key = conversation_key(q);
    std::pair<size_t, long> side{0, -1};
    bool fresh = true;
    {
        std::lock_guard<std::mutex> lock(long_mu);
        if (const auto c = long_cache.find(key); c != long_cache.end()) side = c->second, fresh = false;
    }
    if (fresh) {
        side.second = prompt_tokens(backends[0].port, q);
        side.first = side.second >= o.long_from ? 1 : 0;
        std::lock_guard<std::mutex> lock(long_mu);
        if (long_cache.size() >= 4096) {  // oldest conversation first
            long_cache.erase(long_order.front());
            long_order.pop_front();
        }
        if (long_cache.emplace(key, side).second) long_order.push_back(key);
    }
    const size_t i = side.first;
    char route_hdr[96];
    std::snprintf(route_hdr, sizeof route_hdr, "%s %ld %s", i ? "long" : "short", side.second, backends[i].device.c_str());
    held = std::make_shared<InFlight>(&inflight[i]);
    ++routed[i];
    forward(backends[i].port, held, -1, id, backends[i].drop_model, backends[i].set_model, q, r);
    r.set_header("X-1bit-Route", route_hdr);  // short or long, the first prompt's tokens, device
    return;
}

long prompt_tokens(int port, const httplib::Request& q) {
    json body;
    try {
        body = json::parse(q.body);
    } catch (const std::exception&) {
        return long(q.body.size() / 4);
    }
    httplib::Client c("127.0.0.1", port);
    c.set_read_timeout(std::chrono::seconds(30));
    std::string prompt;
    if (body.contains("prompt") && body["prompt"].is_string()) {
        prompt = body["prompt"].get<std::string>();
    } else if (auto t = c.Post("/apply-template", q.body, "application/json"); t && t->status == 200) {
        try {
            prompt = json::parse(t->body).value("prompt", "");
        } catch (const std::exception&) {
        }
    }
    if (prompt.empty()) return long(request_state(q).size() / 4);
    if (auto t = c.Post("/tokenize", json{{"content", prompt}}.dump(), "application/json"); t && t->status == 200) {
        try {
            const json tok = json::parse(t->body);
            if (tok.contains("tokens") && tok["tokens"].is_array()) return long(tok["tokens"].size());
        } catch (const std::exception&) {
        }
    }
    return long(prompt.size() / 4);
}